/**
 * Main DataCrawler to get lots of stuff to index...;-)
 */
package complet.app.io.datacrawler;

import java.io.IOException;
import java.util.List;
import java.util.logging.Level;
import java.util.regex.Pattern;
import net.htmlparser.jericho.*;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.url.WebURL;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;

public class DataCrawler extends WebCrawler {

    Pattern filters = Pattern.compile(".*(\\.(css|js|bmp|gif|jpe?g"
            + "|png|tiff?|mid|mp2|mp3|mp4" + "|wav|avi|mov|mpeg|ram|m4v|pdf"
            + "|rm|smil|wmv|swf|wma|zip|rar|gz))$");
    final String filename = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\DataCollector\\Completion_DataCollector\\DataFiles\\crawled\\crawled_blogger_1.txt";
    String index_read = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\DataCollector\\Completion_DataCollector\\DataFiles\\crawled\\crawled_blogger_1_index.txt";
    String index_write = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\DataCollector\\Completion_DataCollector\\DataFiles\\crawled\\crawled_blogger_1_index.txt";
    StringBuilder sb_index = new StringBuilder();

    // Constructor
    public DataCrawler() {
        // Get index
        File file = new File(index_read);
        BufferedReader in = null;
        BufferedReader in_count = null;
        try {
            in = new BufferedReader(new FileReader(file));
            in_count = new BufferedReader(new FileReader(file));
        } catch (FileNotFoundException ex) {
            System.err.println("Index to read not found!");
            return;
        }
        String line;
        try {
            while ((line = in_count.readLine()) != null) {
                sb_index.append(line);
            }
        } catch (IOException e) {
            System.err.println("Loading index failed");
            return;
        }
    }

    public boolean shouldVisit(WebURL url) {
        if (url != null) {
            String href = url.getURL().toLowerCase();

            if (filters.matcher(href).matches()) {
                return false;

            }
            if (sb_index.indexOf(url.getURL()) > -1) {
                return false;

            }
            if (href.contains(".blogger.de")) {
                return true;

            }
        }
        return false;
    }

    public void visit(Page page) {
        if (page != null) {
            int docid = page.getWebURL().getDocid();
            String url = page.getWebURL().getURL();
            String text = page.getHTML();

            Source source = new Source(text);
            Element content = source.getElementById("center");

            if (content != null) {
                StringBuilder sb = new StringBuilder();
                List<Element> elements = content.getAllElements("div");
                //elements = content.getChildElements();
                //elements = content.getAllElements();

                for (int i = 0; i
                        < elements.size(); i++) {
                    Element e = elements.get(i);
                    String s = e.toString().trim();


                    if (s.length() >= 25) {
                        s = s.substring(0, 25);


                    }
                    if ((s.contains("<div>") || s.contains("storyBody")) && !(s.contains("dayHeader") || s.contains("pagelink") || s.contains("storyTitle") || s.contains("storyDate") || s.contains("\"small\""))) {
                        TextExtractor ex = e.getTextExtractor();
                        ex.setConvertNonBreakingSpaces(true);
                        ex.setIncludeAttributes(false);

                        sb.append(escapeString(ex.toString()));


                    }
                }

                if (!sb.toString().isEmpty()) {
                    try {
                        FileWriter file = new FileWriter(filename, true);
                        BufferedWriter out = new BufferedWriter(file);
                        out.write(sb.toString() + ".\r\n");
                        out.close();

                    } catch (IOException ex) {
                        java.util.logging.Logger.getLogger(DataCrawler.class.getName()).log(Level.SEVERE, null, ex);
                        System.err.println(
                                "ERROR: Couldn't open file - Invoke app_terminate()");
                        System.exit(
                                1);
                    }
                    if (!index_write.isEmpty()) {
                        try {
                            FileWriter file = new FileWriter(index_write, true);
                            BufferedWriter out = new BufferedWriter(file);
                            out.write(url + "\r\n");
                            out.close();
                        } catch (IOException ex) {
                            System.err.println("Couldn't write index...");
                        }
                    }

                    System.out.println("Docid: " + docid);
                    System.out.println("URL: " + url);
                    System.out.println("Save Text length: " + sb.toString().length());
                    System.out.println("=============");

                }
            }
        }
    }

    private String escapeString(String s) {
        s = s.replace("Ä", "Ae");
        s = s.replace("Ö", "Oe");
        s = s.replace("Ü", "Ue");
        s = s.replace("ä", "Ae");
        s = s.replace("ö", "Oe");
        s = s.replace("ü", "Ue");
        s = s.replace("ß", "ss");
        s = s.replace("<br>", "\r\n");
        s = s.replace("</br>", "\r\n");
        s = s.replaceAll("(http://|www\\.)[a-zA-Z0-9\\-\\.]+\\.(com|org|net|mil|edu|COM|ORG|NET|MIL|EDU|de|DE|biz|BIZ)", "");


        return s;

    }
}
